{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(7.3891, grad_fn=<ExpBackward>)\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor(2.0,requires_grad=True)\n",
    "b = a.exp()\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input = torch.ones([2,2],requires_grad=False)\n",
    "w1 = torch.tensor(2.0,requires_grad = True)\n",
    "w2 = torch.tensor(3.0,requires_grad = True)\n",
    "w3 = torch.tensor(3.0,requires_grad = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1 = input * w1\n",
    "l2 = l1+w2\n",
    "l3 = l1*w3\n",
    "l4 = l2*l3\n",
    "loss = l4.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.) tensor(21.) None\n"
     ]
    }
   ],
   "source": [
    "print(w1.data,w1.grad,w1.grad_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[2., 2.],\n",
      "        [2., 2.]]) None <MulBackward0 object at 0x1080ab2b0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhaomingming/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/tensor.py:746: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  warnings.warn(\"The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad \"\n"
     ]
    }
   ],
   "source": [
    "print(l1.data,l1.grad,l1.grad_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(30.) None <MeanBackward0 object at 0x1081c8da0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhaomingming/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/tensor.py:746: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  warnings.warn(\"The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad \"\n"
     ]
    }
   ],
   "source": [
    "print(loss.data,loss.grad,loss.grad_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 叶子张量 leaf tensor:反向传播时，只保留属性requires_grad和is_leaf为真的张量得导数。\n",
    "# requires_grad为真，is_leaf为假时，此张量得导数作为中间结果用于计算叶子张量得导师。\n",
    "# requires_grad is False,is_leaf is False,次张量参与导数计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "a = torch.ones([2,2],requires_grad=True)\n",
    "print(a.is_leaf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "b = a+2 \n",
    "print(b.is_leaf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 因为b不是用户创建的，是通过计算生成的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 叶子张量的作用：节省内存或者显存 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 叶子节点的grad_fn都为空\n",
    "# 非叶子节点点grad_fn都不为空\n",
    "# 果我们想保留中间变量的导数，该怎么操作？\n",
    "### 通过使用tensor.retain_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.)\n",
      "tensor([[0.2500, 0.2500],\n",
      "        [0.2500, 0.2500]])\n",
      "tensor([[5.2500, 5.2500],\n",
      "        [5.2500, 5.2500]])\n"
     ]
    }
   ],
   "source": [
    "loss = l4.mean()\n",
    "l1.retain_grad()\n",
    "l4.retain_grad()\n",
    "loss.retain_grad()\n",
    "loss.backward()\n",
    "print(loss.grad)\n",
    "print(l4.grad)\n",
    "print(l1.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 如果我们只想进行debug,只需要输出中间变量的导数信息，而不需要保存他们，我们还可以使用tensor.register_hook,如下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss grad: tensor(1.)\n",
      "l4 grad tensor([[0.2500, 0.2500],\n",
      "        [0.2500, 0.2500]])\n",
      "l1 grad: tensor([[5.2500, 5.2500],\n",
      "        [5.2500, 5.2500]])\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/zhaomingming/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/tensor.py:746: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  warnings.warn(\"The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad \"\n"
     ]
    }
   ],
   "source": [
    "loss2 = l4.mean()\n",
    "l1.register_hook(lambda grad:print(\"l1 grad:\",grad))\n",
    "l4.register_hook(lambda grad:print(\"l4 grad\",grad))\n",
    "loss2.register_hook(lambda grad:print(\"loss grad:\",grad))\n",
    "loss2.backward()\n",
    "print(loss2.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 可以看到，是先打印loss的grade,后打印l4,l1得grad得\n",
    "# 并且在最后print(loss2.grad)得时候\n",
    "# 打印出了none\n",
    "# 这说明loss.grad在print完之后，就被清除掉了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pytorch 中，Hook的作用非常大"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# inplace 操作:inplace operation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在不更改变量的内存地址的情况下，直接修改变量的值。就叫inplace操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4527517632\n",
      "4527516768\n",
      "4985982728\n",
      "4985982728\n"
     ]
    }
   ],
   "source": [
    "a  = torch.tensor([3.0,1.0])\n",
    "print(id(a))\n",
    "a = a.exp() # 不是inplace\n",
    "print(id(a))\n",
    "b=[1,2]\n",
    "print(id(b))\n",
    "b[0]=10# 是inplace\n",
    "print(id(b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 以上id有变化的不属于inplace操作,id没有变化的，才是inplace操作"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pytorch 怎么监测tensor发生了inplace操作？通过tensor._version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [2]], which is output 0 of AddBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-0dc1d20245ec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_version\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \"\"\"\n\u001b[0;32m--> 198\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     98\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     99\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [2]], which is output 0 of AddBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True)."
     ]
    }
   ],
   "source": [
    "a = torch.tensor([1.0,3.0],requires_grad=True)\n",
    "b = a+2\n",
    "print(b._version)\n",
    "\n",
    "loss = (b * b).mean()\n",
    "b[0] = 1000.0\n",
    "print(b._version)\n",
    "\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [2]], which is output 0 of AddBackward0, is at version 1; expected version 0 instead."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 每次tensor进行inplace时，_version的值就会加1，在正向传播过程中，求导系统记录的b的version是0，但是反向传播过程中，求导系统发现b的version变成了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对于requires_grad=True的叶子节点的值，在求梯度之前，是不允许修改的。\n",
    "# 类似一下应用："
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RuntimeError: leaf variable has been moved into the graph interior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([10.,  5.,  2.,  3.], requires_grad=True) True\n",
      "tensor([10.,  5.,  2.,  3.], requires_grad=True) 4527514248\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([10.,5.,2.,3.],requires_grad=True)\n",
    "print(a,a.is_leaf)\n",
    "print(a,id(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0., 0., 0., 0.], grad_fn=<CopySlices>) False\n",
      "tensor([0., 0., 0., 0.], grad_fn=<CopySlices>) 4527514248\n"
     ]
    }
   ],
   "source": [
    "a[:]=0\n",
    "a.add_(10.)\n",
    "print(a,a.is_leaf)\n",
    "print(a,id(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-ce4728b1e158>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mloss3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mloss3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    196\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \"\"\"\n\u001b[0;32m--> 198\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda2/envs/python3.6.7/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     98\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     99\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Trying to backward through the graph a second time, but the buffers have already been freed. Specify retain_graph=True when calling backward the first time."
     ]
    }
   ],
   "source": [
    "loss3 = (a*a).mean()\n",
    "loss3.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # 在使用a[:]=0时，实际上是用inplace操作把一个叶子节点变成了非叶子节点了，这样的话，导数就不会被保存了，就变成none了。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 也就是说，在backward之前，我们要想修改叶子节点，必须按照一定的规则"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 方法一 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([10.,  5.,  2.,  3.], requires_grad=True) True 4527548960\n",
      "tensor([5., 5., 5., 5.], requires_grad=True) True 4527548960\n",
      "tensor([2.5000, 2.5000, 2.5000, 2.5000])\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([10.,5.,2.,3.],requires_grad=True)\n",
    "print(a,a.is_leaf,id(a))\n",
    "a.data.fill_(5.)\n",
    "print(a,a.is_leaf,id(a))\n",
    "\n",
    "loss5 = (a*a).mean()\n",
    "loss5.backward()\n",
    "print(a.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 方法二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([10.,  5.,  2.,  3.], requires_grad=True) True\n",
      "tensor([10., 10., 10., 10.], requires_grad=True) True\n",
      "tensor([5., 5., 5., 5.])\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([10.,5.,2.,3.],requires_grad=True)\n",
    "print(a,a.is_leaf)\n",
    "with torch.no_grad():\n",
    "    a[:]=10.\n",
    "print(a,a.is_leaf)\n",
    "loss = (a*a).mean()\n",
    "loss.backward()\n",
    "print(a.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 这里，第一种方法用fill来修改，第二种方法用with torch.no_grad()来修改，其实都是找到了一个变量，这个变量与a共享了内存，但这个变量得requires_grad是False "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 比如,a.data.fill_(5.)时，a.data就是这个中间变量，不但与a共享了内存，而且这个变量是不需要requires_grad的\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# with torch.no_grad() 是不进行grad相关操作。 "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
